
!conda install -qq -y -c conda-forge gdown

Retrieving notices: ...working... done
Collecting package metadata (current_repodata.json): ...working... done
Solving environment: ...working... 
The environment is inconsistent, please check the package plan carefully
The following packages are causing the inconsistency:

  - conda-forge/noarch::bleach==6.0.0=pyhd8ed1ab_0
  - conda-forge/noarch::plotly==5.14.1=pyhd8ed1ab_0
  - conda-forge/noarch::python-slugify==8.0.1=pyhd8ed1ab_1
  - conda-forge/noarch::qtpy==2.3.1=pyhd8ed1ab_0
  - conda-forge/linux-64::sip==6.7.9=py310hc6cd4ac_0
  - conda-forge/noarch::tqdm==4.65.0=pyhd8ed1ab_1
  - conda-forge/linux-64::watchdog==3.0.0=py310hff52083_0
  - conda-forge/noarch::nltk==3.8.1=pyhd8ed1ab_0
  - conda-forge/noarch::flask==2.3.2=pyhd8ed1ab_0
  - conda-forge/noarch::importlib_metadata==6.6.0=hd8ed1ab_0
  - conda-forge/noarch::jsonschema==4.17.3=pyhd8ed1ab_0
  - conda-forge/linux-64::pyqt5-sip==12.11.0=py310heca2aa9_3
  - conda-forge/noarch::pytest==7.3.1=pyhd8ed1ab_0
  - conda-forge/noarch::pytoolconfig==1.2.5=pyhd8ed1ab_0
  - conda-forge/noarch::qdarkstyle==3.1=pyhd8ed1ab_0
  - conda-forge/noarch::qtawesome==1.2.3=pyhd8ed1ab_0
  - conda-forge/linux-64::black==23.3.0=py310hff52083_1
  - conda-forge/noarch::dask-core==2023.4.1=pyhd8ed1ab_0
  - conda-forge/noarch::flask-cors==3.0.10=pyhd8ed1ab_0
  - conda-forge/noarch::jupyter_events==0.6.3=pyhd8ed1ab_0
  - conda-forge/noarch::jupyter_ydoc==0.2.3=pyhd8ed1ab_0
  - conda-forge/noarch::pylint==2.17.4=pyhd8ed1ab_0
  - conda-forge/noarch::rope==1.8.0=pyhd8ed1ab_0
  - conda-forge/noarch::jupyter_client==8.2.0=pyhd8ed1ab_0
  - conda-forge/linux-64::keyring==23.13.1=py310hff52083_0
  - conda-forge/noarch::nbformat==5.8.0=pyhd8ed1ab_0
  - conda-forge/noarch::python-lsp-server==1.7.2=pyhd8ed1ab_0
  - conda-forge/noarch::distributed==2023.4.1=pyhd8ed1ab_0
  - conda-forge/noarch::nbclient==0.7.4=pyhd8ed1ab_0
  - conda-forge/noarch::pyls-spyder==0.4.0=pyhd8ed1ab_0
  - conda-forge/linux-64::pyqt==5.15.7=py310hab646b1_3
  - conda-forge/noarch::python-lsp-black==1.2.1=pyhd8ed1ab_0
  - conda-forge/noarch::requests==2.29.0=pyhd8ed1ab_0
  - conda-forge/noarch::cookiecutter==2.1.1=pyh6c4a22f_0
  - conda-forge/noarch::ipykernel==6.23.0=pyh210e3f2_0
  - conda-forge/noarch::nbconvert-core==7.4.0=pyhd8ed1ab_0
  - conda-forge/noarch::pooch==1.7.0=pyha770c72_3
  - conda-forge/linux-64::pyqtwebengine==5.15.7=py310h1165ae2_3
  - conda-forge/linux-64::requests-kerberos==0.12.0=py310hff52083_4
  - conda-forge/noarch::s3transfer==0.6.1=pyhd8ed1ab_0
  - conda-forge/noarch::sphinx==7.0.0=pyhd8ed1ab_0
  - conda-forge/noarch::ipywidgets==8.0.6=pyhd8ed1ab_0
  - conda-forge/noarch::jupyter_console==6.6.3=pyhd8ed1ab_0
  - conda-forge/noarch::jupyter_server==2.5.0=pyhd8ed1ab_0
  - conda-forge/noarch::nbconvert-pandoc==7.4.0=pyhd8ed1ab_0
  - conda-forge/noarch::numpydoc==1.5.0=pyhd8ed1ab_0
  - conda-forge/noarch::qtconsole-base==5.4.3=pyha770c72_0
  - conda-forge/linux-64::scipy==1.10.1=py310h8deb116_2
  - conda-forge/noarch::sphinxcontrib-websupport==1.2.4=pyhd8ed1ab_1
  - conda-forge/noarch::spyder-kernels==2.4.3=unix_pyhd8ed1ab_0
  - conda-forge/linux-64::astropy==5.2.2=py310h0a54255_0
  - conda-forge/noarch::bokeh==3.1.1=pyhd8ed1ab_0
  - conda-forge/noarch::jupyter_server_fileid==0.9.0=pyhd8ed1ab_0
  - conda-forge/noarch::jupyterlab_server==2.22.1=pyhd8ed1ab_0
  - conda-forge/linux-64::matplotlib-base==3.7.1=py310he60537e_0
  - conda-forge/noarch::nbconvert==7.4.0=pyhd8ed1ab_0
  - conda-forge/noarch::notebook-shim==0.2.3=pyhd8ed1ab_0
  - conda-forge/noarch::patsy==0.5.3=pyhd8ed1ab_0
  - conda-forge/linux-64::pytables==3.8.0=py310hde6a235_1
  - conda-forge/noarch::qtconsole==5.4.3=pyhd8ed1ab_0
  - conda-forge/linux-64::scikit-learn==1.2.2=py310h41b6a48_1
  - conda-forge/noarch::dask==2023.4.1=pyhd8ed1ab_0
  - conda-forge/noarch::jupyter_server_ydoc==0.8.0=pyhd8ed1ab_0
  - conda-forge/linux-64::matplotlib==3.7.1=py310hff52083_0
  - conda-forge/noarch::nbclassic==1.0.0=pyhb4ecaf3_1
  - conda-forge/linux-64::scikit-image==0.20.0=py310h9b08913_1
  - conda-forge/noarch::seaborn-base==0.12.2=pyhd8ed1ab_0
  - conda-forge/linux-64::shap==0.41.0=py310h769672d_0
  - conda-forge/linux-64::spyder==5.4.3=py310hff52083_0
  - conda-forge/linux-64::statsmodels==0.14.0=py310h278f3c1_1
  - conda-forge/noarch::notebook==6.5.4=pyha770c72_0
  - conda-forge/noarch::seaborn==0.12.2=hd8ed1ab_0
  - conda-forge/linux-64::jupyter==1.0.0=py310hff52083_8
  - conda-forge/noarch::jupyterlab==3.6.3=pyhd8ed1ab_0
  - conda-forge/noarch::hdijupyterutils==0.20.5=pyh1a96a4e_0
  - conda-forge/noarch::autovizwidget==0.20.5=pyh1a96a4e_0
  - conda-forge/noarch::sparkmagic==0.20.5=pyhd8ed1ab_0
done

## Package Plan ##

  environment location: /home/ec2-user/anaconda3/envs/python3

  added / updated specs:
    - gdown


The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    botocore-1.29.165          |     pyhd8ed1ab_0         6.0 MB  conda-forge
    docutils-0.20.1            |  py310hff52083_0         704 KB  conda-forge
    gdown-4.7.1                |     pyhd8ed1ab_0          19 KB  conda-forge
    importlib-metadata-6.7.0   |     pyha770c72_0          25 KB  conda-forge
    importlib_metadata-6.7.0   |       hd8ed1ab_0           9 KB  conda-forge
    python-slugify-8.0.1       |     pyhd8ed1ab_0          17 KB  conda-forge
    sphinx-7.0.1               |     pyhd8ed1ab_0         1.2 MB  conda-forge
    ------------------------------------------------------------
                                           Total:         8.0 MB

The following NEW packages will be INSTALLED:

  botocore           conda-forge/noarch::botocore-1.29.165-pyhd8ed1ab_0 
  colorama           conda-forge/noarch::colorama-0.4.6-pyhd8ed1ab_0 
  docutils           conda-forge/linux-64::docutils-0.20.1-py310hff52083_0 
  gdown              conda-forge/noarch::gdown-4.7.1-pyhd8ed1ab_0 
  importlib-metadata conda-forge/noarch::importlib-metadata-6.7.0-pyha770c72_0 
  packaging          conda-forge/noarch::packaging-23.1-pyhd8ed1ab_0 
  pyyaml             conda-forge/linux-64::pyyaml-6.0-py310h5764c6d_5 
  urllib3            conda-forge/noarch::urllib3-1.26.15-pyhd8ed1ab_0 

The following packages will be UPDATED:

  importlib_metadata                       6.6.0-hd8ed1ab_0 --> 6.7.0-hd8ed1ab_0 
  openssl                                  3.1.0-hd590300_3 --> 3.1.1-hd590300_1 
  sphinx                                 7.0.0-pyhd8ed1ab_0 --> 7.0.1-pyhd8ed1ab_0 

The following packages will be DOWNGRADED:

  python-slugify                         8.0.1-pyhd8ed1ab_1 --> 8.0.1-pyhd8ed1ab_0 


Preparing transaction: ...working... done
Verifying transaction: ...working... done
Executing transaction: ...working... done


!gdown 1ewcD6wshP6eHAAbHRELlITJlb7l6b9uC
!unzip -q *.zip
!rm *.zip
!rm -rf __MACOSX/

Downloading...
From (uriginal): https://drive.google.com/uc?id=1ewcD6wshP6eHAAbHRELlITJlb7l6b9uC
From (redirected): https://drive.google.com/uc?id=1ewcD6wshP6eHAAbHRELlITJlb7l6b9uC&confirm=t&uuid=34ed55c6-2ba4-44d5-af19-d0c915867317
To: /home/ec2-user/SageMaker/1DCA.zip
100%|█████████████████████████████████████████| 110M/110M [00:00<00:00, 163MB/s]


!pwd

/home/ec2-user/SageMaker


!mv 1DCA/* /home/ec2-user/SageMaker/
!rm -rf 1DCA


import csv

# Open the input file
with open('master_list_of_files.csv', 'r') as input_file:
    reader = csv.reader(input_file)

    # Read the data rows
    data = [row for row in reader]

# Add the header
header = ['path1', 'path2', 'path3']
data_with_header = [header] + data

# Write the data with header to a new CSV file
with open('paths.csv', 'w', newline='') as output_file:
    writer = csv.writer(output_file)
    writer.writerows(data_with_header)

# print("Header added successfully!")

!rm master_list_of_files.csv


#@title Sort csv file

import csv
import re

def sort_csv_by_first_column(csv_file):
    with open(csv_file, 'r') as file:
        reader = csv.reader(file)
        data = list(reader)
        headers = data[0]  # Get the headers
        data = data[1:]  # Skip the first row (headers)

        sorted_data = sorted(data, key=lambda row: extract_numeric_part(row[0]))

    new_csv_file = 'sorted_' + csv_file
    with open(new_csv_file, 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerow(headers)  # Write the headers back to the sorted file
        writer.writerows(sorted_data)

    print(f"Sorted data has been written to {new_csv_file}.")

def extract_numeric_part(text):
    numeric_part = re.findall(r'\d+', text)  # Extract numeric part using regex
    if numeric_part:
        return int(numeric_part[0])
    else:
        return 0

# Usage example
csv_filename = 'paths.csv'
sort_csv_by_first_column(csv_filename)

Sorted data has been written to sorted_paths.csv.


import csv

def select_rows(input_file, output_file, start_row, end_row):
    with open(input_file, 'r', newline='') as f_in:
        reader = csv.reader(f_in)
        rows = list(reader)

        headers = rows[0]
        selected_rows = rows[start_row-1:end_row]

    with open(output_file, 'w', newline='') as f_out:
        writer = csv.writer(f_out)
        writer.writerow(headers)
        writer.writerows(selected_rows)

# Specify the range of rows you want to select
m = 5000  # Start row
n = 5500  # End row

input_file = 'sorted_paths.csv'
output_file = f'sorted_paths_{m}_{n}.csv'

select_rows(input_file, output_file, m, n)


#@title Display all results


import pandas as pd
from IPython.display import display, HTML
import base64
# Read the CSV file
df = pd.read_csv(output_file)

# Apply CSS style for zooming
display(HTML('''
<style>
.zoomable-image {
  transition: transform 0.2s;
}

.zoomable-image:hover {
  transform: scale(1.5);
}
</style>
'''))

# Iterate over each row in the DataFrame
for _, row in df.iterrows():
    html_string = ''
    html_string += '<div style="display: flex; flex-direction: row; margin: 10px;">'

    # Iterate over each image path in the row
    for col in ['path3', 'path2', 'path1']:
        image_path = row[col]
        image_data = base64.b64encode(open(image_path, 'rb').read()).decode('utf-8')

        html_string += '<div style="display: flex; flex-direction: column; text-align: center; margin-right: 10px;">'
        html_string += f'<p>Image: {image_path}</p>'

        # Add CSS class 'zoomable-image' to path2 image
        if col == 'path2':
            html_string += f'<img src="data:image/svg+xml;base64,{image_data}" class="zoomable-image" width="400" height="400" />'
        else:
            html_string += f'<img src="data:image/svg+xml;base64,{image_data}" width="400" height="400" />'

        html_string += '</div>'

    html_string += '</div>'

    display(HTML(html_string))